{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5e599549-929a-418e-a7dd-546163418854",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "原始数据     RT @Amila #Test\n",
      "Tom's newly listed Co  &amp; Mary's unlisted     Group to supply tech for nlTK.\n",
      "h $TSLA $AAPL https:// t.co/x34afsfQsh \n",
      "\n",
      "去除特殊标签后的:     RT  \n",
      "Tom's newly listed Co   Mary's unlisted     Group to supply tech for nlTK.\n",
      "h $TSLA $AAPL https:// t.co/x34afsfQsh \n",
      "\n",
      "去除价值符号后的:     RT  \n",
      "Tom's newly listed Co   Mary's unlisted     Group to supply tech for nlTK.\n",
      "h   https:// t.co/x34afsfQsh \n",
      "\n",
      "去除超链接后的:     RT  \n",
      "Tom's newly listed Co   Mary's unlisted     Group to supply tech for nlTK.\n",
      "h    \n",
      "\n",
      "去除专门名词缩写后:       \n",
      "Tom' newly listed    Mary' unlisted     Group  supply tech for nlTK.\n",
      "    \n",
      "\n",
      "去除空格、单引号和句号后的： Tom newly listed Mary unlisted Group supply tech for nlTK  \n",
      "\n",
      "分词结果: ['Tom', 'newly', 'listed', 'Mary', 'unlisted', 'Group', 'supply', 'tech', 'for', 'nlTK'] \n",
      "\n",
      "去除停用词后结果: ['Tom', 'newly', 'listed', 'Mary', 'unlisted', 'Group', 'supply', 'tech', 'nlTK'] \n",
      "\n",
      "过滤后: Tom newly listed Mary unlisted Group supply tech nlTK\n"
     ]
    }
   ],
   "source": [
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.text import Text\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import re\n",
    "#输入数据\n",
    "s = '    RT @Amila #Test\\nTom\\'s newly listed Co  &amp; Mary\\'s unlisted     Group to supply tech for nlTK.\\nh $TSLA $AAPL https:// t.co/x34afsfQsh'\n",
    "\n",
    "#指定停用词\n",
    "cache_engliish_stopwords = stopwords.words('english')\n",
    "\n",
    "def text_clean(text):\n",
    "    print('原始数据',text,'\\n')\n",
    "\n",
    "    #去除HTML标签（e.g. &amp;）\n",
    "    text_no_special_entities = re.sub(r'\\&\\w*;|#\\w*|@\\w*','',text)\n",
    "    print('去除特殊标签后的:',text_no_special_entities,'\\n')\n",
    "\n",
    "    #去除一些价值符号\n",
    "    text_no_tickers = re.sub(r'\\$\\w*','',text_no_special_entities)\n",
    "    print('去除价值符号后的:',text_no_tickers,'\\n')\n",
    "\n",
    "    #去除超链接\n",
    "    text_no_hyperlinks = re.sub(r'https?:\\/\\/.*\\/\\w*','',text_no_tickers)\n",
    "    print('去除超链接后的:',text_no_hyperlinks,'\\n')\n",
    "\n",
    "    #去除专有名称\n",
    "    text_no_small_words = re.sub(r'\\b\\w{1,2}\\b','',text_no_hyperlinks)\n",
    "    print('去除专门名词缩写后:',text_no_small_words,'\\n')\n",
    "\n",
    "    #去除多余的空格、单引号和句号\n",
    "    text_no_whitespace = re.sub(r'[\\s\\s\\'.]+',' ',text_no_small_words)\n",
    "    text_no_whitespace = text_no_whitespace.lstrip(' ')\n",
    "    print('去除空格、单引号和句号后的：',text_no_whitespace,'\\n')\n",
    "\n",
    "    #分词\n",
    "    tokens = word_tokenize(text_no_whitespace)\n",
    "    print('分词结果:',tokens,'\\n')\n",
    "\n",
    "    #去除停用词\n",
    "    list_no_stopwords = [i for i in tokens if i not in cache_engliish_stopwords]\n",
    "    print('去除停用词后结果:',list_no_stopwords,'\\n')\n",
    "\n",
    "    text_filtered = ' '.join(list_no_stopwords)\n",
    "    print('过滤后:',text_filtered)\n",
    "\n",
    "text_clean(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77254843-4181-4f1d-b6ce-74de55f50d94",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
